/*
 * @(#)BncParser.java	1.0 08/01/07
 *
 * Copyright 2007 Fabio Gasparetti. All rights reserved.
 */

package org.tabularium.text.nlp;

import java.io.*;
import java.util.*;

import org.tabularium.text.nlp.wordnet.PartOfSpeech;;

/**
 * Parser for the British National Corpus (BNC) file.
 * <p>
 * Useful for building a {@see CollecitonFrequency} object. 
 * <p>
 * Retrieve the collection from {@link ftp://ftp.itri.bton.ac.uk/bnc/all.num.o5}
 * 
 * @author Fabio Gasparetti
 * @version 1.0, 08/01/07
 */ 
public class BncParser implements CollectionFrequencyParser {
	// the first word is the POS class (see PartOfSpeech class), followed by the POS attributed used by BNC	
	protected static final String[] VERB = { "VERB", "VBB", "VBD", "VBG", "VBI", "VBN", "VBZ", "VDB", "VDD", "VDG", "VDI", "VDN", "VDZ", "VHB", "VHD", "VHG", "VHI", "VHN", "VHZ", "VM0", "VVB", "VVD", "VVG", "VVI", "VVN", "VVZ", "VVD-VVN", "VVN-VVD" };
	protected static final String[] UNC = { "UNC", "UNC", "AJ0-AV0", "AV0-AJ0", "AJ0-NN1", "NN1-AJ0", "AJ0-VVD", "VVD-AJ0", "AJ0-VVG", "VVG-AJ0", "AJ0-VVN", "VVN-AJ0", "AVP-PRP", "PRP-AVP", "AVQ-CJS", "CJS-AVQ", "CJS-PRP", "PRP-CJS", "CJT-DT0", "DT0-CJT", "CRD-PNI", "PNI-CRD", "NN1-VVB", "VVB-NN1", "NN1-VVG", "VVG-NN1", "NN2-VVZ", "VVZ-NN2" };
	protected static final String[] SUBST = { "NOUN", "SUBST", "NN0", "NN1", "NN2", "NP0", "ONE", "ZZ0", "NN1-NP0", "NP0-NN1" };
	protected static final String[] STOP = { "STOP", "POS", "PUL", "PUN", "PUQ", "PUR" };
	protected static final String[] PRON = { "PRONOUN", "PRON", "DPS", "DTQ", "EX0", "PNI", "PNP", "PNQ", "PNX" };
	protected static final String[] PREP = { "PREPOSITION", "PREP", "PRF", "PRP", "TO0" };
	protected static final String[] INTERJ = { "INTERJECTION", "INTERJ", "ITJ" };
	protected static final String[] CONJ = { "CONJUCTION", "CONJ", "CJC", "CJS", "CJT" };
	protected static final String[] ART = { "ARTICLE", "ART", "AT0" };
	protected static final String[] ADV = { "ADVERB", "ADV", "AV0", "AVP", "AVQ", "XX0" };
	protected static final String[] ADJ = { "ADJECTIVE", "ADJ", "AJ0", "AJC", "AJS", "CRD", "DT0", "ORD" };
	
	protected static final String[][] POS_CLASSES = { VERB, UNC, SUBST, STOP, PRON, PREP, INTERJ, CONJ, ART, ADV, ADJ };
	
	protected Map posConversionMap = null;
	protected Stemmer stemmer = null;
	
	public BncParser() {
		posConversionMap = new HashMap();
		for (int i = 0; i < POS_CLASSES.length; i++) {
			for (int j = 0; j < POS_CLASSES[i].length; j++) {
				posConversionMap.put(POS_CLASSES[i][j], POS_CLASSES[i][0]);				
			}
		}
	}
	
	public BncParser(Stemmer stem) {
 		this();
		this.stemmer = stem;
	}
	
	public List read(String filename) throws IOException {
		Map m = new TreeMap(); 
		FileReader reader = new FileReader(filename);
		BufferedReader bufReader = new BufferedReader(reader);
		CollectionFrequency.TermFreq tf, temp;
		CollectionFrequency.TermFreq first;
		String line;
		String term;
		String[] splits;
		// 1st line -> parsing # docs
		if ((line = bufReader.readLine()) == null) 
			return null;
		splits = line.split(" ");
		if (splits.length != 4) 
			return null;
		first = new CollectionFrequency.TermFreq(splits[1].toUpperCase(), 0, Integer.parseInt(splits[3]));
		// rest of the file
		while ((line = bufReader.readLine()) != null) {
			splits = line.split(" ");
			if (splits.length != 4) 
				return null;
			String pos = (String)posConversionMap.get(splits[2].toUpperCase());
			if (pos == null)
				pos = splits[2]; // keep original if not found
			term = splits[1];
			if (stemmer != null) { 
				Set s = stemmer.getRoots(term, PartOfSpeech.toInteger(pos));
				if (s != null)
					term = (String)s.iterator().next();
			}
			tf = new CollectionFrequency.TermFreq(term.toUpperCase(), PartOfSpeech.toInteger(pos), Integer.parseInt(splits[0]));
			//System.out.println(tf.term+" "+tf.pos);
			temp = (CollectionFrequency.TermFreq)m.get(tf);
			if (temp != null) {
				temp.freq += tf.freq;
			} else {
				m.put(tf, tf);
			}
		}
		bufReader.close();
		reader.close();
		// map -> list
		ArrayList l = new ArrayList();
		l.add(first);
		Iterator iter = m.keySet().iterator();
		while (iter.hasNext()) {
			l.add(iter.next());
		}
		return l;
	}
	
}
